package com.darrenchan.spark.sql

import org.apache.spark.sql.SparkSession

/**
  * Parquet文件操作
  */
object ParquetApp {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder().master("local[2]").appName("ParquetApp").getOrCreate()

    //业务逻辑
    //spark.read.parquet("users.parquet")等价
    //spark.read.load("users.parquet")等价，默认数据格式就是parquet
    //spark.read.format("parquet").option("path", "users.parquet").load()
    val usersDF = spark.read.format("parquet").load("users.parquet")

    usersDF.printSchema()

    usersDF.show()


    val selectDF = usersDF.select(usersDF.col("name"), usersDF.col("favorite_numbers"))
//    selectDF.write.json("D:\\hello\\sparkout")
    selectDF.write.json("file:///Users/chenchi03/Desktop")

    spark.stop()
  }
}
